package com.algo.url;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.BufferedReader;
import java.io.PrintStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Properties;

import edu.stanford.nlp.ie.crf.CRFClassifier;
import edu.stanford.nlp.ling.CoreLabel;

/**
 *	this class can segment titles grabbed from url, and generate url to keywords hashmap
 *	<br/>
 *	@author:lujianfeng@miaozhen.com
 */

public class SpidKeyWordsMapFactory{
        private String urlTitle;	// = "/home/algo/lujianfeng/urltitle/htmlresult.csv";
        private String urlKeyWords;	// = "/home/algo/lujianfeng/urltitle/urlkeywords";
        private String segmenterData = "/home/algo/lujianfeng/urltitle/stanford-segmenter-2014-08-27/data";
        private String encoding = "utf-8";

        public SpidKeyWordsMapFactory(){}
        public SpidKeyWordsMapFactory(String in, String out){
                urlTitle = in;
		urlKeyWords = out;
        }
        public Map<String, String> getSpidKeyWordsMap() throws IOException{
                File file = new File(urlKeyWords);
                if(!file.exists())
                        segmentTitle();
		printMemoryInfo("-----------befor hashmap--------------");
		Map<String, String> map = new HashMap<String, String>();
		BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), encoding));
		String line;
		while((line = br.readLine()) != null){
			int pos = line.indexOf("\t");
			if(pos < 0) continue;
			map.put(line.substring(0, pos), line.substring(pos+1));
		}
		printMemoryInfo("-------------after hashmap-------------");
                return map;
        }
	public void printMemoryInfo(String msg){
		System.gc();
		Runtime rt = Runtime.getRuntime();
		long totalMem = rt.totalMemory();
		long freeMem = rt.freeMemory();
		System.out.println(msg);
		System.out.println("total mem:" + totalMem);
		System.out.println("free mem" + freeMem);
		System.out.println("used mem:" + (totalMem-freeMem));
	}
        public void segmentTitle() throws IOException{
                //PrintStream ps = new PrintStream(new FileOutputStream(urlKeyWords), true, encoding);
                //System.setOut(ps);
                Properties props = new Properties();
                props.setProperty("sighanCorporaDict", segmenterData);
                props.setProperty("serDictionary", segmenterData + "/dict-chris6.ser.gz");
                props.setProperty("inputEncoding", encoding);
                props.setProperty("sighanPostProcessing", "true");

                CRFClassifier<CoreLabel> segmenter = new CRFClassifier<CoreLabel>(props);
                segmenter.loadClassifierNoExceptions(segmenterData + "/ctb.gz", props);


                PrintStream ps = new PrintStream(new FileOutputStream(urlKeyWords), true, encoding);
                try{
                        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(urlTitle), encoding));
                        try{
                                String line;
                                while((line = br.readLine()) != null){
                                        /*String[] kv = line.split("\\t");
                                        assert(kv.length == 2);
					if(kv.length < 2){
						System.out.println("kv.length < 2:" + kv);
						continue;
					}
                                        List<String> segmented = segmenter.segmentString(kv[1]);
                                        ps.print(kv[0] + "\t");
                                        for(String word : segmented)
                                                ps.print(word+"|");
                                        ps.println();*/
                                        int pos = line.indexOf("\t");
                                        if(pos < 0){
                                                System.out.println("pos < 0:" + line);
                                                continue;
                                        }
                                        List<String> segmented = segmenter.segmentString(line.substring(pos+1));
                                        ps.print(line.substring(0, pos) + "\t");
                                        for(String word : segmented)
                                                ps.print(word+"|");
                                        ps.println();
                                }
                        }finally{
                                br.close();
                        }
                }finally{
                    ps.close();
           	}
    	}
	public static void main(String[] args)throws IOException{
		new SpidKeyWordsMapFactory(args[0], args[1]).getSpidKeyWordsMap();
	}
}

